# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from evny.items import EvnyItem
import hashlib
import time
import datetime
import re
from elasticsearch import Elasticsearch

es = Elasticsearch(['127.0.0.1:9200'])

class ForumSpider(CrawlSpider):
    name = 'forum'
    allowed_domains = ['www.eyny.com']
    start_urls = ['https://www.eyny.com/forum-1724-9JO5ETH6.html']
    rules = (
        Rule(LinkExtractor(allow=r'forum-1724-(.*)\.html',restrict_xpaths='//*[@id="threadlist"]//a'), follow=True),
        Rule(LinkExtractor(allow=r'thread-(.*)-(.*)-(.*)\.html',restrict_xpaths='//*[@id="threadlist"]'), callback='parse_item'),
    )

    def parse_item(self, response):
        # first_page 第一页第一条为主贴
        first_page = int(re.findall(r'thread-.*-(.*)-.*\.html',response.url)[0])
        # 固定source
        source = 'eyny'
        # 帖子id
        first_page_url = re.sub('thread-(.*)-(.*)-(.*).html',r'thread-\1-1-\3.html',response.url)
        first_page_url_id = get_md5_id(first_page_url)
        title = response.xpath('//a[@id="thread_subject"]/text()').extract_first()
        post_list = response.xpath('//div[@id="postlist"]/div[starts-with(@id,"post_")]')
        if first_page == 1:
            # 查看数
            buddha_operation = int(response.xpath('//div[@id="postlist"]/table[1]/tr//span[2]/text()').extract_first())
            # 回复数
            comments = int(response.xpath('//div[@id="postlist"]/table[1]/tr//span[last()]/text()').extract_first())
        count = len(post_list)
        for i in range(count):
            post = post_list[i]
            id = post.xpath('./@id').extract_first().replace('post_','')
            # 用户区域
            user = post.xpath('./table[@id]/tr[1]/td[1]')[0]
            # 内容文本区域
            content = post.xpath('./table[@id]/tr[1]/td[2]')[0]
            # 作者昵称
            author = user.xpath('.//div[@class="authi"]/a[starts-with(@href,"space-uid-")]/text()').extract_first()
            # 作者等级
            author_grade = user.xpath('.//a[starts-with(@href,"home.php?mod=spacecp&ac=usergroup&gid")]/font/text()').extract_first()
            # 发布时间
            datetime_str = content.xpath('.//div[@class="authi"]/em[@id]/text()').extract_first()
            if datetime_str == '發表於 ':
                datetime_str = content.xpath('.//div[@class="authi"]/em[@id]/span/@title').extract_first()
            # 转换时间戳
            datetime,timestamp = trans_date_ts(datetime_str)
            # 内容
            context = ''.join(content.xpath('.//td[starts-with(@id,"postmessage_")]//text()').extract())
            if not context:
                continue
            item = EvnyItem(title=title,id=first_page_url_id,author=author,author_grade=author_grade,context=context,time=datetime,timestamp=timestamp)
            if first_page == 1 and i == 1:
                index='post'
                item['comments'] = comments
                item['buddha_operation'] = buddha_operation
                index_id = first_page_url_id
            else:
                index = 'post2'
                index_id = id
            es.index(index=index,doc_type='text',id=index_id,body=dict(item))
            print(item)

def date2ts(date):
    try:
        return int(time.mktime(time.strptime(date, '%Y-%m-%d')))
    except:
        return int(time.mktime(time.strptime(date, '%Y-%m-%d %H:%M:%S')))
def ts2date(ts):
    return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ts))
def trans_date_ts(date_str):
    date = date_str.replace('發表於 ','').replace('半小時前','30 分鐘前').replace(' PM','').replace(' AM','')
    # 转成时间戳
    now_ts = time.time()
    if '分鐘前' in date:
        minute = int(date.rstrip(' 分鐘前'))
        ts = now_ts - minute * 60
        date = ts2date(ts)
    elif '小時前' in date:
        hour = int(date.rstrip(' 小時前'))
        ts = now_ts - hour * 3600
        date = ts2date(ts)
    elif '天前' in date:
        hour = int(date.rstrip(' 天前'))
        ts = now_ts - hour * 86400
        date = ts2date(ts)
    elif '昨天' in date:
        new_time = date.lstrip('昨天')
        date = get_before_date(1) + new_time + ":00"
    elif '前天' in date:
        new_time = date.lstrip('前天')
        date = get_before_date(2) + new_time + ":00"
    else:
        format_time = time.strptime(date, '%Y-%m-%d %H:%M')
        date = time.strftime('%Y-%m-%d %H:%M:%S', format_time)
    if "PM" in date_str:
        date = ts2date(date2ts(date)+43200)
    return date, date2ts(date)

def get_before_date(date_cha):
    # date_cha为当天日期的前几天,返回前几天的日期
    # 如当天2018-7-7,date_cha=1,则返回2018-7-6
    today = datetime.date.today()
    oneday = datetime.timedelta(days=date_cha)
    newday = today - oneday
    return str(newday)

def get_md5_id(url):
    md5 = hashlib.md5()
    md5.update(url.encode())
    return md5.hexdigest()

if __name__ == '__main__':
    d,timestamp = trans_date_ts('昨天 11:19 PM')
    print(d,timestamp)